﻿using System;
using System.Collections;
using System.ComponentModel;
using System.Data;
using HtmlAgilityPack;
using System.IO;
using System.Xml;
using System.Text;
using ICSharpCode.SharpZipLib.Zip;
using SDT.CodeSystem;

using SDT.CrawlSystem.Servers.Filters;
using SDT.FileSystem;
using SDT.Libraries.Convertor;
using SDT.Libraries.Impersonator;
using SDT.ServiceSystem;
using System.Text.RegularExpressions;

namespace SDT.CrawlSystem.Servers
{
    
    public sealed partial class WebCrawler : WorkerThread
    {
        #region Remove Tag
        public string CleanHtml(string s)
        {
            //HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
            //doc.LoadHtml(s);
            //remove tag introdue trong cafef
            return RemoveTagIntrodue(s, @"/div/div[1]");
            //remove other tag
            //RemoveTag(doc, tag);
            //RemoveAttribute(doc, att);
            //StringWriter writer = new StringWriter();
            //doc.Save(writer);

            //string result = writer.ToString();
            //if (string.IsNullOrEmpty(result)) result = s;
            
            //writer.Close();
            
            //return result;
        }
        private void RemoveAttribute(HtmlAgilityPack.HtmlDocument doc, string attributes)
        {
            StringBuilder sb = new StringBuilder("//*[");
            char[] splitter = { '|' };
            string[] atts = attributes.Split(splitter);
            for (int i = 0; i < atts.Length; i++)
            {
                if (i > 0)
                    sb.Append("|");
                sb.Append("@");
                sb.Append(atts[i]);
            }
            sb.Append("]");
            string XPath = sb.ToString();
            HtmlNodeCollection nodes = doc.DocumentNode.SelectNodes(XPath);
            if (nodes == null)
                return;
            foreach (HtmlNode node in nodes)
            {
                foreach (string attName in atts)
                {
                    HtmlAttribute att = node.Attributes[attName];
                    if (att != null)
                        node.Attributes.Remove(att);
                }
            }
        }
        private void RemoveTag(HtmlAgilityPack.HtmlDocument doc, string tags)
        {
            HtmlNodeCollection nodes = doc.DocumentNode.SelectNodes("//" + tags);
            if (nodes == null)
                return;
            foreach (HtmlNode node in nodes)
            {
                if (node.ParentNode != null)
                    node.ParentNode.RemoveChild(node);
            }
        }
        #endregion
      
        #region Strip tags

        private  string ReplaceFirst(string haystack, string needle, string replacement)
        {
            int pos = haystack.IndexOf(needle);
            if (pos < 0) return haystack;
            return haystack.Substring(0, pos) + replacement + haystack.Substring(pos + needle.Length);
        }

        private  string ReplaceFirstContent(string haystack, string needle, string replacement)
        {
            int pos = haystack.IndexOf(needle);
            if (pos < 0) return haystack;
            return haystack.Substring(0, pos) + replacement + haystack.Substring(pos + needle.Length);
        }

        private  string ReplaceAll(string haystack, string needle, string replacement)
        {
            int pos;
            // Avoid a possible infinite loop
            if (needle == replacement) return haystack;
            while ((pos = haystack.IndexOf(needle)) > 0)
                haystack = haystack.Substring(0, pos) + replacement + haystack.Substring(pos + needle.Length);
            return haystack;
        }

        public  bool IsExceptionTags(string Tag, string[] ExceptionTags)
        {
            foreach (string itm in ExceptionTags)
            {
                if (Tag.Contains(itm))
                    return true;
            }
            return false;
        }

        public string StripTags(string Input, string allowtag)
        {
            char[] splitter = { '|' };
            string[] AllowedTags = allowtag.Split(splitter);

            Regex StripHTMLExp = new Regex(@"(<\/?[^>]+>)");
            string Output = Input;

            foreach (Match Tag in StripHTMLExp.Matches(Input))
            {
                string HTMLTag = Tag.Value.ToLower();
                bool IsAllowed = false;

                foreach (string AllowedTag in AllowedTags)
                {
                    int offset = -1;

                    // Determine if it is an allowed tag
                    // "<tag>" , "<tag " and "</tag"
                    if (offset != 0) offset = HTMLTag.IndexOf('<' + AllowedTag + '>');
                    if (offset != 0) offset = HTMLTag.IndexOf('<' + AllowedTag + ' ');
                    if (offset != 0) offset = HTMLTag.IndexOf("</" + AllowedTag);

                    // If it matched any of the above the tag is allowed
                    if (offset == 0)
                    {
                        IsAllowed = true;
                        break;
                    }
                }
                // Remove tags that are not allowed
                if (!IsAllowed) Output = ReplaceFirst(Output, Tag.Value, "");

            }

            //remove unwanted

            return Output;
        }

        public static string StripStartAndEndWith(string start, string end, string html)
        {
            try
            {
                int intStart = html.IndexOf(start, 0);
                int intEnd = html.IndexOf(end, start.Length + intStart);
                int intCount = intEnd + end.Length - intStart;
                return html.Remove(intStart, intCount);
            }
            catch
            {
                return string.Empty;
            }

        }


        /// <summary>
        /// Author: TungNX
        /// Created Date:
        /// Remove tag introdue
        /// </summary>
        /// <param name="doc"></param>
        /// <param name="tags"></param>
        //private void RemoveTagIntrodue(HtmlAgilityPack.HtmlDocument doc, string tags, string att, string attvalue)
        //{
        //    HtmlNodeCollection nodes = doc.DocumentNode.SelectNodes("//" + tags);
        //    if (nodes == null)
        //        return;
        //    foreach (HtmlNode node in nodes)
        //    {
        //        if (node.ParentNode != null && node.Attributes[att].Value.ToString().Contains(attvalue))
        //        {
        //            nodes.Remove(node);           // Modifile by VietLV (date:01/06/2011)
        //            break;                        // Description: change "node.ParentNode.RemoveChild(node);" by as "nodes.Remove(node); then break"    
        //            // node.ParentNode.RemoveChild(node);
        //        }
        //    }
        //}
        public static string RemoveTagIntrodue(string strHtml, string xPath)
        {
            HtmlDocument htmlSnippet = new HtmlDocument();

            htmlSnippet.LoadHtml(strHtml);
            var resultRemoveLink = htmlSnippet.DocumentNode.SelectNodes(xPath);
            if (resultRemoveLink != null)
            {
                foreach (HtmlNode link in htmlSnippet.DocumentNode.SelectNodes(xPath))
                {
                    link.RemoveAll();
                }
                return htmlSnippet.DocumentNode.WriteContentTo();
            }
            return htmlSnippet.DocumentNode.WriteContentTo();
        }
        /// <summary>
        /// Author: TungNX
        /// Created Date : 
        /// Remove thẻ introdue trong cafef
        /// </summary>
        /// <param name="s"></param>
        /// <param name="tag"></param>
        /// <param name="att"></param>
        /// <returns></returns>
        //public string RemoveTagIntrodue(string s, string tag, string att, string attvalue)
        //{
        //    HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
        //    doc.LoadHtml(s);
        //    RemoveTagIntrodue(doc, tag, att, attvalue);

        //    StringWriter writer = new StringWriter();
        //    doc.Save(writer);

        //    string result = writer.ToString();
        //    if (string.IsNullOrEmpty(result)) result = s;

        //    writer.Close();

        //    return result;
        //}


        public string StripTagsAndAttributes(string Input, string AllowedTags)
        {
            /* Remove all unwanted tags first */
            string Output = StripTags(Input, AllowedTags);

            /* Lambda functions */
            MatchEvaluator HrefMatch = m => m.Groups[1].Value + "href..;,;.." + m.Groups[2].Value;
            MatchEvaluator ClassMatch = m => m.Groups[1].Value + "class..;,;.." + m.Groups[2].Value;
            MatchEvaluator UnsafeMatch = m => m.Groups[1].Value + m.Groups[4].Value;

            /* Allow the "href" attribute */
            Output = new Regex("(<a.*)href=(.*>)").Replace(Output, HrefMatch);

            /* Allow the "class" attribute */
            Output = new Regex("(<a.*)class=(.*>)").Replace(Output, ClassMatch);

            /* Remove unsafe attributes in any of the remaining tags */
            //Output = new Regex(@"(<.*) .*=(\'|\""|\w)[\w|.|(|)]*(\'|\""|\w)(.*>)").Replace(Output, UnsafeMatch);

            /* Return the allowed tags to their proper form */
            Output = ReplaceAll(Output, "..;,;..", "=");

            return Output;
        }
        public string RemoveHTML(string htmlString)
        {
            string regularExpresion = "<[^<>]*>";
            Regex regex = new Regex(regularExpresion, RegexOptions.IgnoreCase | RegexOptions.Multiline);
            return regex.Replace(htmlString, "").Trim();
        }

        #endregion
       

    }
}